﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using HtmlAgilityPack;

namespace HttpCrawler.Engine
{
    public class HttpCrawlerEngine : IHttpCrawlerEngine
    {
        private const string _IE8UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 1.1.4322; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; InfoPath.2)";

        private const int _TwoMinuteTimeout = 1000 * 60 * 2;

        private readonly Uri _BaseUrl;

        private readonly int _TimeoutInMilliseconds;

        private readonly string _UserAgent;

        private readonly ConcurrentDictionary<Uri, object> _VisitedUrls = new ConcurrentDictionary<Uri, object>();

        private readonly ConcurrentDictionary<string, object> _InvalidUrls = new ConcurrentDictionary<string, object>();

        private readonly IHttpCrawlerContentProcessor[] _Processors;

        private readonly IHttpCrawlerContentPersister _Persister;

        private readonly IHttpCrawlerUriFollowDeterminer _UriFollowDeterminer;

        private readonly IHttpCrawlerEngineLogger _Logger;

        public HttpCrawlerEngine(
            Uri baseUrl,
            string userAgent,
            IHttpCrawlerContentPersister persister,
            IHttpCrawlerUriFollowDeterminer uriFollowDeterminer,
            IHttpCrawlerEngineLogger logger,
            int timeoutInMilliseconds,
            params IHttpCrawlerContentProcessor[] processors)
        {
            _BaseUrl = baseUrl;
            _UserAgent = userAgent;
            _Persister = persister;
            _UriFollowDeterminer = uriFollowDeterminer;
            _Logger = logger;
            _TimeoutInMilliseconds = timeoutInMilliseconds;
            _Processors = processors;
        }

        public HttpCrawlerEngine(Uri baseUrl)
            : this(
                baseUrl,
                _IE8UserAgent,
                new DoNothingHttpCrawlerContentPersister(),
                new SimilarOriginHttpCrawlerUriFollowDeterminer(),
                new DoNothingHttpCrawlerEngineLogger(),
                _TwoMinuteTimeout,
                new HtmlHttpCrawlerContentProcessor(),
                new CssHttpCrawlerContentProcessor())
        {
        }

        /// <summary>
        /// Allows for parsing form and p elements' contents.
        /// </summary>
        /// <remarks>
        /// Needs to be run before any activity involving HtmlDocument.
        /// </remarks>
        protected static void SetupParsing()
        {
            HtmlNode.ElementsFlags.Remove("form");
            HtmlNode.ElementsFlags.Remove("p");

            HtmlNode.ElementsFlags.Add("form", HtmlElementFlag.CanOverlap);
            HtmlNode.ElementsFlags.Add("p", HtmlElementFlag.Closed);
        }

        public IEnumerable<Uri> Crawl()
        {
            var started = DateTime.Now;

            _Logger.Log(String.Format("Crawl Starting: {0} ({1})", _BaseUrl, started));

            SetupParsing();

            var queue = new ConcurrentQueue<KeyValuePair<Uri, Uri>>();

            queue.Enqueue(_BaseUrl, null);

            ParallelHelper.While(
                () => queue.Count > 0,
                () =>
                {
                    KeyValuePair<Uri, Uri> urlToProcess;

                    if (queue.TryDequeue(out urlToProcess))
                    {
                        var urls = ProcessUrl(urlToProcess.Key, urlToProcess.Value);

                        foreach (var item in urls)
                        {
                            if (!_VisitedUrls.ContainsKey(item))
                            {
                                queue.Enqueue(item, urlToProcess.Key);
                            }
                        }
                    }
                });

            _Logger.Log(String.Format("Crawl Finished: {0} ({1} [{2}])", _BaseUrl, DateTime.Now, DateTime.Now - started));

            return _VisitedUrls.Keys;
        }

        protected IEnumerable<Uri> ProcessUrl(Uri url, Uri referrerUrl)
        {
            if (ShouldNotProcessUrl(url))
            {
                return Enumerable.Empty<Uri>();
            }

            AddVisitedUrl(url);

            var foundUrls = new HashSet<string>();

            try
            {
                var request = CreateRequest(url, referrerUrl);

                using (var response = (HttpWebResponse)request.GetResponse())
                {
                    using (var crawlerContent = HttpCrawlerContentFactory.Create(url, referrerUrl, response))
                    {
                        foundUrls.AddRange(ProcessContent(crawlerContent));

                        PersistContent(crawlerContent);
                    }
                }
            }
            catch (Exception ex)
            {
                _Logger.DownloadFailure(url, ex);

                return Enumerable.Empty<Uri>();
            }

            _Logger.DownloadSuccess(url);

            AddVisitedUrl(url);

            var urlsToDownload = ProcessFoundUrls(url, foundUrls);

            return urlsToDownload;
        }

        private bool ShouldNotProcessUrl(Uri url)
        {
            return url == null || !url.IsAbsoluteUri || WasVisited(url);
        }

        private IEnumerable<Uri> ProcessFoundUrls(Uri url, IEnumerable<string> foundUrls)
        {
            var urlsToDownload = new HashSet<Uri>();

            foreach (var foundUrl in foundUrls)
            {
                if (!IsKnownInvalidUrl(foundUrl))
                {
                    Uri foundUri;

                    if (Uri.IsWellFormedUriString(foundUrl, UriKind.Absolute))
                    {
                        foundUri = new Uri(foundUrl);
                    }
                    else if (Uri.IsWellFormedUriString(foundUrl, UriKind.Relative))
                    {
                        foundUri = new Uri(url, new Uri(foundUrl, UriKind.Relative));
                    }
                    else
                    {
                        AddInvalidUrl(foundUrl);

                        continue;
                    }

                    if (_UriFollowDeterminer.Follow(_BaseUrl, foundUri))
                    {
                        urlsToDownload.Add(foundUri);
                    }
                }
            }

            return urlsToDownload;
        }

        private HttpWebRequest CreateRequest(Uri url, Uri referrerUrl)
        {
            var request = (HttpWebRequest)WebRequest.Create(url);

            request.AllowAutoRedirect = true;
            request.MaximumAutomaticRedirections = 3;
            request.UserAgent = _UserAgent;
            request.Timeout = _TimeoutInMilliseconds;

            if (referrerUrl != null)
            {
                request.Referer = referrerUrl.ToString();
            }

            return request;
        }

        private void PersistContent(HttpCrawlerContent crawlerContent)
        {
            try
            {
                _Persister.PersistContent(crawlerContent);
                _Logger.PersistContentSuccess(_Persister, crawlerContent);
            }
            catch (Exception ex)
            {
                _Logger.PersistContentFailed(_Persister, crawlerContent, ex);
            }
        }

        private IEnumerable<string> ProcessContent(HttpCrawlerContent crawlerContent)
        {
            var urlCandidates = new HashSet<string>();

            foreach (var processor in _Processors)
            {
                if (processor.CanProcess(crawlerContent))
                {
                    try
                    {
                        var foundUrlCandidates = processor.Process(crawlerContent);

                        if (foundUrlCandidates != null)
                        {
                            urlCandidates.AddRange(foundUrlCandidates);
                        }

                        _Logger.ProcessSuccess(processor, crawlerContent);
                    }
                    catch (Exception ex)
                    {
                        _Logger.ProcessFailed(processor, crawlerContent, ex);
                    }
                }
            }

            return urlCandidates;
        }

        private bool IsKnownInvalidUrl(string foundUrl)
        {
            return _InvalidUrls.ContainsKey(foundUrl);
        }

        private bool WasVisited(Uri url)
        {
            return _VisitedUrls.ContainsKey(url);
        }

        private void AddVisitedUrl(Uri url)
        {
            if (WasVisited(url))
            {
                return;
            }

            _VisitedUrls.AddOrUpdate(url, (object)null, (k, v) => v);
        }

        private void AddInvalidUrl(string url)
        {
            _InvalidUrls.AddOrUpdate(url, (object)null, (k, v) => v);
        }
    }
}
